#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import json
import logging
import sys
import os
import importlib.util
from datetime import datetime

# 添加项目路径到系统路径
sys.path.append(os.path.dirname(os.path.abspath(__file__)))

# 动态导入 prometheus-agent 模块
spec = importlib.util.spec_from_file_location("prometheus_agent", "/Users/yaoyafei/Desktop/python/prom/prometheus-agent.py")
if spec is not None and spec.loader is not None:
    prometheus_agent_module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(prometheus_agent_module)
    InspectionAgent = prometheus_agent_module.InspectionAgent
else:
    raise ImportError("无法导入 prometheus-agent 模块")

# 导入飞书通知模块
spec_lark = importlib.util.spec_from_file_location("lark", "/Users/yaoyafei/Desktop/python/prom/lark.py")
if spec_lark is not None and spec_lark.loader is not None:
    lark_module = importlib.util.module_from_spec(spec_lark)
    spec_lark.loader.exec_module(lark_module)
    FeishuNotifier = lark_module.FeishuNotifier
else:
    raise ImportError("无法导入 lark 模块")

# 配置日志
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("ClusterInspection")

class ClusterInspector:
    def __init__(self, config_path="/Users/yaoyafei/Desktop/python/prom/config.json"):
        """初始化集群巡检器"""
        with open(config_path, 'r') as f:
            self.config = json.load(f)
        
        self.agent = InspectionAgent(self.config["prometheus_url"])
        self.notifier = FeishuNotifier(self.config["feishu_webhook"])
        
    def collect_metrics(self):
        """采集指标数据"""
        logger.info("开始采集集群指标数据...")
        metrics = self.agent.collect_metrics()
        logger.info(f"采集完成，共获取 {len(metrics)} 类指标")
        return metrics
    
    def analyze_cluster_resources(self, metrics):
        """分析集群资源使用情况"""
        analysis = {
            "timestamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
            "summary": {
                "total_nodes": 0,
                "cpu_usage": {"avg": 0, "max": 0, "min": 0, "high_usage_nodes": []},
                "memory_usage": {"avg": 0, "max": 0, "min": 0, "high_usage_nodes": []},
                "disk_usage": {"avg": 0, "max": 0, "min": 0, "high_usage_nodes": []},
                "jvm_usage": {"avg": 0, "max": 0, "min": 0, "high_usage_instances": []},
                "health_check": {"failed": [], "total": 0},
                "site_probing": {"failed": [], "total": 0},
                "alerts": {"active": [], "total": 0},
                # 添加中间件状态
                "middleware": {
                    "mysql": {"status": "unknown", "instances": []},
                    "redis": {"status": "unknown", "instances": []},
                    "rabbitmq": {"status": "unknown", "instances": []}
                }
            },
            "details": {
                "node_cpu": [],
                "node_memory": [],
                "node_disk": [],
                "jvm_memory": [],
                "app_healthcheck": [],
                "site_xunjian": [],
                "alerts": [],
                # 添加中间件详细信息
                "mysql_instances": [],
                "redis_instances": [],
                "rabbitmq_instances": []
            }
        }
        
        # 分析CPU使用率
        self._analyze_cpu_usage(metrics, analysis)
        
        # 分析内存使用率
        self._analyze_memory_usage(metrics, analysis)
        
        # 分析磁盘使用率
        self._analyze_disk_usage(metrics, analysis)
        
        # 分析JVM内存使用率
        self._analyze_jvm_usage(metrics, analysis)
        
        # 分析应用健康检查
        self._analyze_health_check(metrics, analysis)
        
        # 分析站点巡检
        self._analyze_site_probing(metrics, analysis)
        
        # 分析中间件状态
        self._analyze_middleware_status(metrics, analysis)
        
        # 分析告警信息（从Prometheus告警API获取）
        self._analyze_alerts(analysis)
        
        return analysis
    
    def _analyze_cpu_usage(self, metrics, analysis):
        """分析CPU使用率"""
        if "node_cpu" in metrics and "data" in metrics["node_cpu"] and "data" in metrics["node_cpu"]["data"]:
            cpu_data = metrics["node_cpu"]["data"]["data"]
            if "result" in cpu_data:
                cpu_values = []
                for series in cpu_data["result"]:
                    if "values" in series and series["values"]:
                        latest_value = float(series["values"][-1][1])
                        instance = series["metric"].get("instance", "unknown")
                        cpu_values.append(latest_value)
                        analysis["details"]["node_cpu"].append({"instance": instance, "usage": latest_value})
                        
                        # 记录高CPU使用率节点
                        if latest_value > 80:
                            analysis["summary"]["cpu_usage"]["high_usage_nodes"].append(
                                {"instance": instance, "usage": latest_value}
                            )
                
                if cpu_values:
                    analysis["summary"]["total_nodes"] = len(cpu_values)
                    analysis["summary"]["cpu_usage"]["avg"] = sum(cpu_values) / len(cpu_values)
                    analysis["summary"]["cpu_usage"]["max"] = max(cpu_values)
                    analysis["summary"]["cpu_usage"]["min"] = min(cpu_values)
    
    def _analyze_memory_usage(self, metrics, analysis):
        """分析内存使用率"""
        if "node_memory" in metrics and "data" in metrics["node_memory"] and "data" in metrics["node_memory"]["data"]:
            memory_data = metrics["node_memory"]["data"]["data"]
            if "result" in memory_data:
                memory_values = []
                for series in memory_data["result"]:
                    if "values" in series and series["values"]:
                        latest_value = float(series["values"][-1][1])
                        instance = series["metric"].get("instance", "unknown")
                        # 提取IP和应用信息
                        ip = series["metric"].get("ip", "unknown")
                        job = series["metric"].get("job", "unknown")
                        memory_values.append(latest_value)
                        analysis["details"]["node_memory"].append({
                            "instance": instance, 
                            "ip": ip,
                            "job": job,
                            "usage": latest_value
                        })
                        
                        # 记录高内存使用率节点
                        if latest_value > 85:
                            analysis["summary"]["memory_usage"]["high_usage_nodes"].append(
                                {
                                    "instance": instance, 
                                    "ip": ip,
                                    "job": job,
                                    "usage": latest_value
                                }
                            )
                
                if memory_values:
                    analysis["summary"]["memory_usage"]["avg"] = sum(memory_values) / len(memory_values)
                    analysis["summary"]["memory_usage"]["max"] = max(memory_values)
                    analysis["summary"]["memory_usage"]["min"] = min(memory_values)
    
    def _analyze_disk_usage(self, metrics, analysis):
        """分析磁盘使用率"""
        if "node_disk" in metrics and "data" in metrics["node_disk"] and "data" in metrics["node_disk"]["data"]:
            disk_data = metrics["node_disk"]["data"]["data"]
            if "result" in disk_data:
                disk_values = []
                for series in disk_data["result"]:
                    if "values" in series and series["values"]:
                        latest_value = float(series["values"][-1][1])
                        device = series["metric"].get("device", "unknown")
                        mountpoint = series["metric"].get("mountpoint", "/")
                        instance = series["metric"].get("instance", "unknown")
                        disk_values.append(latest_value)
                        analysis["details"]["node_disk"].append({
                            "device": device, 
                            "mountpoint": mountpoint, 
                            "instance": instance,
                            "usage": latest_value
                        })
                        
                        # 记录高磁盘使用率节点
                        if latest_value > 90:
                            analysis["summary"]["disk_usage"]["high_usage_nodes"].append({
                                "device": device,
                                "mountpoint": mountpoint,
                                "instance": instance,
                                "usage": latest_value
                            })
                
                if disk_values:
                    analysis["summary"]["disk_usage"]["avg"] = sum(disk_values) / len(disk_values)
                    analysis["summary"]["disk_usage"]["max"] = max(disk_values)
                    analysis["summary"]["disk_usage"]["min"] = min(disk_values)
    
    def _analyze_jvm_usage(self, metrics, analysis):
        """分析JVM内存使用率"""
        if "jvm_memory" in metrics and "data" in metrics["jvm_memory"] and "data" in metrics["jvm_memory"]["data"]:
            jvm_data = metrics["jvm_memory"]["data"]["data"]
            if "result" in jvm_data:
                jvm_values = []
                for series in jvm_data["result"]:
                    if "values" in series and series["values"]:
                        latest_value = float(series["values"][-1][1])
                        area = series["metric"].get("area", "unknown")
                        instance = series["metric"].get("instance", "unknown")
                        # 提取更多应用信息
                        job = series["metric"].get("job", "unknown")
                        application = series["metric"].get("application", job)
                        service = series["metric"].get("service", "unknown")
                        ip = series["metric"].get("ip", "unknown")
                        jvm_values.append(latest_value)
                        analysis["details"]["jvm_memory"].append({
                            "area": area, 
                            "instance": instance, 
                            "app_name": application,
                            "service": service,
                            "ip": ip,
                            "usage": latest_value
                        })
                        
                        # 记录高JVM内存使用率实例
                        if latest_value > 85:
                            analysis["summary"]["jvm_usage"]["high_usage_instances"].append(
                                {
                                    "area": area, 
                                    "instance": instance, 
                                    "app_name": application,
                                    "service": service,
                                    "ip": ip,
                                    "usage": latest_value
                                }
                            )
                
                if jvm_values:
                    analysis["summary"]["jvm_usage"]["avg"] = sum(jvm_values) / len(jvm_values)
                    analysis["summary"]["jvm_usage"]["max"] = max(jvm_values)
                    analysis["summary"]["jvm_usage"]["min"] = min(jvm_values)
    
    def _analyze_health_check(self, metrics, analysis):
        """分析应用健康检查"""
        if "app_healthcheck" in metrics and "data" in metrics["app_healthcheck"] and "data" in metrics["app_healthcheck"]["data"]:
            health_data = metrics["app_healthcheck"]["data"]["data"]
            if "result" in health_data:
                total_checks = 0
                failed_checks = 0
                for series in health_data["result"]:
                    if "values" in series and series["values"]:
                        total_checks += 1
                        latest_value = float(series["values"][-1][1])
                        job = series["metric"].get("job", "unknown")
                        instance = series["metric"].get("instance", "unknown")
                        # 提取更多应用信息
                        application = series["metric"].get("service", job)  # 使用service作为应用名
                        service = series["metric"].get("type", "unknown")   # 使用type作为服务类型
                        site = series["metric"].get("site", "unknown")      # 使用site作为URL
                        ip = "unknown"
                        # 从site中提取IP
                        if site != "unknown":
                            try:
                                # 从URL中提取IP地址
                                import re
                                ip_match = re.search(r'http://(\d+\.\d+\.\d+\.\d+)', site)
                                if ip_match:
                                    ip = ip_match.group(1)
                            except:
                                pass
                        
                        analysis["details"]["app_healthcheck"].append({
                            "job": job,
                            "instance": instance,
                            "application": application,
                            "service": service,
                            "ip": ip,
                            "site": site,
                            "status": "healthy" if latest_value == 1 else "unhealthy",
                            "value": latest_value
                        })
                        
                        # 记录失败的健康检查
                        if latest_value != 1:
                            failed_checks += 1
                            analysis["summary"]["health_check"]["failed"].append({
                                "job": job,
                                "instance": instance,
                                "application": application,
                                "service": service,
                                "ip": ip,
                                "site": site,
                                "value": latest_value
                            })
                
                analysis["summary"]["health_check"]["total"] = total_checks
    
    def _analyze_site_probing(self, metrics, analysis):
        """分析站点巡检"""
        if "site_xunjian" in metrics and "data" in metrics["site_xunjian"] and "data" in metrics["site_xunjian"]["data"]:
            site_data = metrics["site_xunjian"]["data"]["data"]
            if "result" in site_data:
                total_probes = 0
                failed_probes = 0
                for series in site_data["result"]:
                    if "values" in series and series["values"]:
                        total_probes += 1
                        latest_value = float(series["values"][-1][1])
                        job = series["metric"].get("job", "unknown")
                        instance = series["metric"].get("instance", "unknown")
                        url = series["metric"].get("instance", "unknown")  # URL在instance字段中
                        
                        analysis["details"]["site_xunjian"].append({
                            "job": job,
                            "instance": instance,
                            "url": url,
                            "status": "success" if latest_value == 1 else "failed",
                            "value": latest_value
                        })
                        
                        # 记录失败的站点巡检
                        if latest_value != 1:
                            failed_probes += 1
                            analysis["summary"]["site_probing"]["failed"].append({
                                "job": job,
                                "instance": instance,
                                "url": url,
                                "value": latest_value
                            })
                
                analysis["summary"]["site_probing"]["total"] = total_probes
    
    def _analyze_middleware_status(self, metrics, analysis):
        """分析中间件状态"""
        # 分析MySQL状态
        if "mysql_up" in metrics and "data" in metrics["mysql_up"] and "data" in metrics["mysql_up"]["data"]:
            mysql_data = metrics["mysql_up"]["data"]["data"]
            if "result" in mysql_data:
                mysql_instances = []
                for series in mysql_data["result"]:
                    if "values" in series and series["values"]:
                        latest_value = float(series["values"][-1][1])
                        instance = series["metric"].get("instance", "unknown")
                        job = series["metric"].get("job", "mysql")
                        
                        mysql_instance = {
                            "instance": instance,
                            "job": job,
                            "status": "up" if latest_value == 1 else "down",
                            "value": latest_value
                        }
                        mysql_instances.append(mysql_instance)
                        
                        # 更新汇总信息
                        if latest_value != 1:
                            analysis["summary"]["middleware"]["mysql"]["status"] = "down"
                            
                # 如果所有实例都正常，则标记为up
                if mysql_instances and analysis["summary"]["middleware"]["mysql"]["status"] != "down":
                    analysis["summary"]["middleware"]["mysql"]["status"] = "up"
                    
                analysis["summary"]["middleware"]["mysql"]["instances"] = mysql_instances
                analysis["details"]["mysql_instances"] = mysql_instances
        
        # 分析Redis状态
        if "redis_up" in metrics and "data" in metrics["redis_up"] and "data" in metrics["redis_up"]["data"]:
            redis_data = metrics["redis_up"]["data"]["data"]
            if "result" in redis_data:
                redis_instances = []
                for series in redis_data["result"]:
                    if "values" in series and series["values"]:
                        latest_value = float(series["values"][-1][1])
                        instance = series["metric"].get("instance", "unknown")
                        job = series["metric"].get("job", "redis")
                        
                        redis_instance = {
                            "instance": instance,
                            "job": job,
                            "status": "up" if latest_value == 1 else "down",
                            "value": latest_value
                        }
                        redis_instances.append(redis_instance)
                        
                        # 更新汇总信息
                        if latest_value != 1:
                            analysis["summary"]["middleware"]["redis"]["status"] = "down"
                            
                # 如果所有实例都正常，则标记为up
                if redis_instances and analysis["summary"]["middleware"]["redis"]["status"] != "down":
                    analysis["summary"]["middleware"]["redis"]["status"] = "up"
                    
                analysis["summary"]["middleware"]["redis"]["instances"] = redis_instances
                analysis["details"]["redis_instances"] = redis_instances
        
        # 分析RabbitMQ状态
        if "rabbitmq_up" in metrics and "data" in metrics["rabbitmq_up"] and "data" in metrics["rabbitmq_up"]["data"]:
            rabbitmq_data = metrics["rabbitmq_up"]["data"]["data"]
            if "result" in rabbitmq_data:
                rabbitmq_instances = []
                for series in rabbitmq_data["result"]:
                    if "values" in series and series["values"]:
                        latest_value = float(series["values"][-1][1])
                        instance = series["metric"].get("instance", "unknown")
                        job = series["metric"].get("job", "rabbitmq")
                        
                        rabbitmq_instance = {
                            "instance": instance,
                            "job": job,
                            "status": "up" if latest_value == 1 else "down",
                            "value": latest_value
                        }
                        rabbitmq_instances.append(rabbitmq_instance)
                        
                        # 更新汇总信息
                        if latest_value != 1:
                            analysis["summary"]["middleware"]["rabbitmq"]["status"] = "down"
                            
                # 如果所有实例都正常，则标记为up
                if rabbitmq_instances and analysis["summary"]["middleware"]["rabbitmq"]["status"] != "down":
                    analysis["summary"]["middleware"]["rabbitmq"]["status"] = "up"
                    
                analysis["summary"]["middleware"]["rabbitmq"]["instances"] = rabbitmq_instances
                analysis["details"]["rabbitmq_instances"] = rabbitmq_instances
    
    def _analyze_alerts(self, analysis):
        """分析告警信息（从Prometheus告警API获取）"""
        try:
            import requests
            # 从配置中获取Prometheus URL
            prometheus_url = self.config["prometheus_url"].rstrip("/")
            
            # 查询活跃告警
            alerts_url = f"{prometheus_url}/api/v1/alerts"
            response = requests.get(alerts_url, timeout=10)
            
            if response.status_code == 200:
                alerts_data = response.json()
                if alerts_data.get("status") == "success":
                    active_alerts = []
                    alerts = alerts_data.get("data", {}).get("alerts", [])
                    
                    for alert in alerts:
                        # 只处理firing状态的告警
                        if alert.get("state") == "firing":
                            alert_data = {
                                "alertname": alert.get("labels", {}).get("alertname", "Unknown"),
                                "instance": alert.get("labels", {}).get("instance", "Unknown"),
                                "severity": alert.get("labels", {}).get("severity", "unknown"),
                                "description": alert.get("annotations", {}).get("description", "No description"),
                                "value": alert.get("value", "N/A"),
                                "state": alert.get("state", "unknown")
                            }
                            # 添加解决方案建议
                            alert_data["solution"] = self._get_alert_solution(alert_data)
                            active_alerts.append(alert_data)
                    
                    analysis["summary"]["alerts"]["total"] = len(active_alerts)
                    analysis["summary"]["alerts"]["active"] = active_alerts
                    analysis["details"]["alerts"] = active_alerts
                else:
                    # 如果无法获取告警数据，使用模拟数据
                    self._use_simulated_alerts(analysis)
            else:
                # 如果无法获取告警数据，使用模拟数据
                self._use_simulated_alerts(analysis)
        except Exception as e:
            logger.warning(f"获取告警数据失败: {e}，使用模拟数据")
            # 如果无法获取告警数据，使用模拟数据
            self._use_simulated_alerts(analysis)
    
    def _get_alert_solution(self, alert):
        """根据告警名称提供解决方案建议"""
        alertname = alert.get("alertname", "")
        solutions = {
            "HighCPUUsage": "检查应用程序负载，考虑水平扩展或优化代码性能。",
            "HighMemoryUsage": "检查内存泄漏，优化JVM参数，考虑增加内存或重启应用。",
            "DiskSpaceLow": "清理磁盘空间，删除不必要的日志文件或临时文件。",
            "InstanceDown": "检查实例是否正常运行，重启服务或排查网络连接问题。",
            "MySQLDown": "检查MySQL服务状态，确保数据库正常运行，查看错误日志。",
            "RedisDown": "检查Redis服务状态，确保缓存服务正常运行。",
            "RabbitMQDown": "检查RabbitMQ服务状态，确保消息队列服务正常运行。",
            "ServiceDown": "检查服务是否正常运行，查看应用日志，重启服务。",
            "SiteProbeFailed": "检查站点是否可访问，排查网络连接或服务状态问题。"
        }
        return solutions.get(alertname, "请根据告警描述手动排查问题。")
    
    def _use_simulated_alerts(self, analysis):
        """使用模拟告警数据（只模拟firing状态的告警）"""
        # 模拟一些firing状态的告警数据
        active_alerts = [
            {
                "alertname": "HighCPUUsage",
                "instance": "172.24.176.111:9100",
                "severity": "warning",
                "description": "CPU使用率超过80%",
                "value": "85.2%",
                "state": "firing"
            },
            {
                "alertname": "HighMemoryUsage",
                "instance": "172.24.177.112:9100",
                "severity": "critical",
                "description": "内存使用率超过90%",
                "value": "92.5%",
                "state": "firing"
            }
        ]
        
        # 为模拟告警添加解决方案
        for alert in active_alerts:
            alert["solution"] = self._get_alert_solution(alert)
        
        analysis["summary"]["alerts"]["total"] = len(active_alerts)
        analysis["summary"]["alerts"]["active"] = active_alerts
        analysis["details"]["alerts"] = active_alerts
    
    def generate_report(self, analysis):
        """生成资源分析报告"""
        summary = analysis["summary"]
        
        report = f"🖥️ 集群资源使用情况分析报告\n"
        report += f"📊 报告时间: {analysis['timestamp']}\n\n"
        
        report += f"📈 集群概览:\n"
        report += f"   • 节点总数: {summary['total_nodes']}\n"
        report += f"   • CPU使用率: 平均{summary['cpu_usage']['avg']:.2f}% (最高{summary['cpu_usage']['max']:.2f}%, 最低{summary['cpu_usage']['min']:.2f}%)\n"
        report += f"   • 内存使用率: 平均{summary['memory_usage']['avg']:.2f}% (最高{summary['memory_usage']['max']:.2f}%, 最低{summary['memory_usage']['min']:.2f}%)\n"
        report += f"   • 磁盘使用率: 平均{summary['disk_usage']['avg']:.2f}% (最高{summary['disk_usage']['max']:.2f}%, 最低{summary['disk_usage']['min']:.2f}%)\n"
        report += f"   • JVM内存使用率: 平均{summary['jvm_usage']['avg']:.2f}% (最高{summary['jvm_usage']['max']:.2f}%, 最低{summary['jvm_usage']['min']:.2f}%)\n"
        report += f"   • 健康检查: {summary['health_check']['total']}个 (失败{len(summary['health_check']['failed'])}个)\n"
        report += f"   • 站点巡检: {summary['site_probing']['total']}个 (失败{len(summary['site_probing']['failed'])}个)\n"
        report += f"   • 活跃告警: {summary['alerts']['total']}个\n"
        
        # 添加中间件状态概览
        mysql_status = "🟢 正常" if summary['middleware']['mysql']['status'] == 'up' else "🔴 异常" if summary['middleware']['mysql']['status'] == 'down' else "⚪ 未知"
        redis_status = "🟢 正常" if summary['middleware']['redis']['status'] == 'up' else "🔴 异常" if summary['middleware']['redis']['status'] == 'down' else "⚪ 未知"
        rabbitmq_status = "🟢 正常" if summary['middleware']['rabbitmq']['status'] == 'up' else "🔴 异常" if summary['middleware']['rabbitmq']['status'] == 'down' else "⚪ 未知"
        
        report += f"   • MySQL状态: {mysql_status}\n"
        report += f"   • Redis状态: {redis_status}\n"
        report += f"   • RabbitMQ状态: {rabbitmq_status}\n\n"
        
        # 资源警告
        warnings = []
        if summary["cpu_usage"]["high_usage_nodes"]:
            warnings.append(f"⚠️ {len(summary['cpu_usage']['high_usage_nodes'])}个节点CPU使用率过高")
        if summary["memory_usage"]["high_usage_nodes"]:
            warnings.append(f"⚠️ {len(summary['memory_usage']['high_usage_nodes'])}个节点内存使用率过高")
        if summary["disk_usage"]["high_usage_nodes"]:
            warnings.append(f"⚠️ {len(summary['disk_usage']['high_usage_nodes'])}个节点磁盘使用率过高")
        if summary["jvm_usage"]["high_usage_instances"]:
            warnings.append(f"⚠️ {len(summary['jvm_usage']['high_usage_instances'])}个实例JVM内存使用率过高")
        if summary["health_check"]["failed"]:
            warnings.append(f"⚠️ {len(summary['health_check']['failed'])}个应用健康检查失败")
        if summary["site_probing"]["failed"]:
            warnings.append(f"⚠️ {len(summary['site_probing']['failed'])}个站点巡检失败")
        if summary["middleware"]["mysql"]["status"] == "down":
            warnings.append(f"⚠️ MySQL服务异常")
        if summary["middleware"]["redis"]["status"] == "down":
            warnings.append(f"⚠️ Redis服务异常")
        if summary["middleware"]["rabbitmq"]["status"] == "down":
            warnings.append(f"⚠️ RabbitMQ服务异常")
        if summary["alerts"]["total"] > 0:
            warnings.append(f"⚠️ {summary['alerts']['total']}个活跃告警")
        
        if warnings:
            report += f"🚨 资源警告:\n"
            for warning in warnings:
                report += f"   • {warning}\n"
            report += "\n"
        else:
            report += "✅ 所有资源使用率均在正常范围内\n\n"
        
        # 集群稳定性评估
        report += f"⚖️ 集群稳定性评估:\n"
        stability_score, stability_desc = self._assess_cluster_stability(summary)
        report += f"   • 稳定性评分: {stability_score}/100\n"
        report += f"   • 健康状况: {stability_desc}\n"
        
        # 根据稳定性评估给出建议
        if stability_score >= 80:
            report += f"   • 建议: 当前集群运行稳定，可继续观察\n"
        elif stability_score >= 60:
            report += f"   • 建议: 集群存在潜在风险，建议及时处理警告项\n"
        elif stability_score >= 40:
            report += f"   • 建议: 集群稳定性较差，建议立即处理异常项\n"
        else:
            report += f"   • 建议: 集群存在严重问题，需要紧急处理\n"
        report += "\n"
        
        # 详细信息（仅显示前5项）
        if summary["cpu_usage"]["high_usage_nodes"]:
            report += f"🖥️ 高CPU使用率节点 (前5):\n"
            for node in summary["cpu_usage"]["high_usage_nodes"][:5]:
                report += f"   • {node['instance']}: {node['usage']:.2f}%\n"
            report += "\n"
            
        if summary["memory_usage"]["high_usage_nodes"]:
            report += f"🧠 高内存使用率节点 (前5):\n"
            for node in summary["memory_usage"]["high_usage_nodes"][:5]:
                # 显示IP和应用信息
                ip_info = f" ({node['ip']})" if node['ip'] != 'unknown' else ""
                job_info = f" [{node['job']}]" if node['job'] != 'unknown' else ""
                full_info = f"{ip_info}{job_info}" if ip_info or job_info else ""
                report += f"   • {node['instance']}{full_info}: {node['usage']:.2f}%\n"
            report += "\n"
            
        if summary["disk_usage"]["high_usage_nodes"]:
            report += f"💾 高磁盘使用率设备 (前5):\n"
            for disk in summary["disk_usage"]["high_usage_nodes"][:5]:
                report += f"   • {disk['instance']} - {disk['device']} ({disk['mountpoint']}): {disk['usage']:.2f}%\n"
            report += "\n"
            
        if summary["jvm_usage"]["high_usage_instances"]:
            report += f"☕ 高JVM内存使用率实例 (前5):\n"
            for jvm in summary["jvm_usage"]["high_usage_instances"][:5]:
                # 显示IP、应用名和服务名
                app_info = jvm['app_name'] if jvm['app_name'] != 'unknown' else ""
                service_info = f" [{jvm['service']}]" if jvm['service'] != 'unknown' else ""
                ip_info = f" ({jvm['ip']})" if jvm['ip'] != 'unknown' else ""
                full_app_info = f" {app_info}{service_info}{ip_info}" if app_info or service_info or ip_info else ""
                report += f"   • {jvm['instance']}{full_app_info} - {jvm['area']}: {jvm['usage']:.2f}%\n"
            # 添加JVM内存使用率过高的解决方案建议
            report += f"     💡 解决方案建议:\n"
            report += f"        • 检查应用程序是否存在内存泄漏\n"
            report += f"        • 优化JVM参数配置，调整堆内存大小\n"
            report += f"        • 分析堆转储文件(Heap Dump)找出内存占用高的对象\n"
            report += f"        • 考虑对应用进行水平扩展\n"
            report += f"        • 重启应用释放内存\n"
            report += "\n"
            
        # 健康检查失败信息
        if summary["health_check"]["failed"]:
            report += f"🏥 健康检查失败 (前5):\n"
            for health in summary["health_check"]["failed"][:5]:
                app_info = health['application'] if health['application'] != 'unknown' else ""
                service_info = f" [{health['service']}]" if health['service'] != 'unknown' else ""
                ip_info = f" ({health['ip']})" if health['ip'] != 'unknown' else ""
                full_app_info = f" {app_info}{service_info}{ip_info}" if app_info or service_info or ip_info else ""
                report += f"   • {health['instance']}{full_app_info} [{health['job']}]: {health['value']}\n"
            report += "\n"
            
        # 站点巡检失败信息
        if summary["site_probing"]["failed"]:
            report += f"🌐 站点巡检失败 (前5):\n"
            for site in summary["site_probing"]["failed"][:5]:
                url_info = f" ({site['url']})" if site['url'] != 'unknown' else ""
                report += f"   • {site['instance']}{url_info} [{site['job']}]: {site['value']}\n"
            report += "\n"
            
        # 中间件状态详情 - 只显示不健康的中间件
        mysql_down_instances = [instance for instance in summary["middleware"]["mysql"]["instances"] if instance['status'] != 'up']
        if mysql_down_instances:
            report += f"🗄️ MySQL异常实例:\n"
            for instance in mysql_down_instances[:5]:
                status_icon = "🔴" if instance['status'] == 'down' else "⚪"
                report += f"   • {status_icon} {instance['instance']} [{instance['job']}]: {instance['status']} ({instance['value']})\n"
            report += "\n"
            
        redis_down_instances = [instance for instance in summary["middleware"]["redis"]["instances"] if instance['status'] != 'up']
        if redis_down_instances:
            report += f" Redis异常实例:\n"
            for instance in redis_down_instances[:5]:
                status_icon = "🔴" if instance['status'] == 'down' else "⚪"
                report += f"   • {status_icon} {instance['instance']} [{instance['job']}]: {instance['status']} ({instance['value']})\n"
            report += "\n"
            
        rabbitmq_down_instances = [instance for instance in summary["middleware"]["rabbitmq"]["instances"] if instance['status'] != 'up']
        if rabbitmq_down_instances:
            report += f"📬 RabbitMQ异常实例:\n"
            for instance in rabbitmq_down_instances[:5]:
                status_icon = "🔴" if instance['status'] == 'down' else "⚪"
                report += f"   • {status_icon} {instance['instance']} [{instance['job']}]: {instance['status']} ({instance['value']})\n"
            report += "\n"
            
        # 活跃告警信息
        if summary["alerts"]["total"] > 0:
            report += f"🔔 活跃告警 (前5):\n"
            for alert in summary["alerts"]["active"][:5]:
                severity_icon = "🔴" if alert['severity'] == 'critical' else "🟡"
                state_info = f" [{alert['state']}]" if alert['state'] != 'unknown' else ""
                report += f"   • {severity_icon} {alert['alertname']}{state_info}: {alert['instance']} - {alert['description']} ({alert['value']})\n"
                # 添加解决方案建议
                if alert.get('solution'):
                    report += f"     💡 解决方案: {alert['solution']}\n"
            report += "\n"
        
        return report
    
    def _assess_cluster_stability(self, summary):
        """评估集群稳定性并返回评分和描述"""
        score = 100  # 满分100分
        
        # 根据各项指标扣分
        # CPU使用率过高扣分
        if summary["cpu_usage"]["high_usage_nodes"]:
            score -= min(len(summary["cpu_usage"]["high_usage_nodes"]) * 2, 20)
        
        # 内存使用率过高扣分
        if summary["memory_usage"]["high_usage_nodes"]:
            score -= min(len(summary["memory_usage"]["high_usage_nodes"]) * 2, 20)
        
        # 磁盘使用率过高扣分
        if summary["disk_usage"]["high_usage_nodes"]:
            score -= min(len(summary["disk_usage"]["high_usage_nodes"]) * 3, 25)
        
        # JVM内存使用率过高扣分
        if summary["jvm_usage"]["high_usage_instances"]:
            score -= min(len(summary["jvm_usage"]["high_usage_instances"]) * 3, 30)
        
        # 健康检查失败扣分
        if summary["health_check"]["failed"]:
            score -= min(len(summary["health_check"]["failed"]) * 5, 30)
        
        # 站点巡检失败扣分
        if summary["site_probing"]["failed"]:
            score -= min(len(summary["site_probing"]["failed"]) * 5, 30)
        
        # 中间件异常扣分
        middleware_issues = 0
        if summary["middleware"]["mysql"]["status"] == "down":
            middleware_issues += 1
        if summary["middleware"]["redis"]["status"] == "down":
            middleware_issues += 1
        if summary["middleware"]["rabbitmq"]["status"] == "down":
            middleware_issues += 1
            
        score -= middleware_issues * 10
        
        # 活跃告警扣分
        if summary["alerts"]["total"] > 0:
            score -= min(summary["alerts"]["total"] * 3, 25)
        
        # 确保分数不低于0
        score = max(score, 0)
        
        # 根据评分给出稳定性描述
        if score >= 90:
            desc = "🟢 非常健康"
        elif score >= 75:
            desc = "🟡 良好"
        elif score >= 60:
            desc = "🟡 一般"
        elif score >= 40:
            desc = "🟠 较差"
        else:
            desc = "🔴 严重问题"
            
        return score, desc
    
    def send_report(self, report):
        """推送报告到飞书"""
        try:
            success = self.notifier.send_message(report)
            if success:
                logger.info("集群资源分析报告已成功推送至飞书")
            else:
                logger.error("集群资源分析报告推送至飞书失败")
        except Exception as e:
            logger.error(f"推送报告时发生错误: {e}")
    
    def run(self):
        """执行完整的集群巡检流程"""
        try:
            # 采集指标
            metrics = self.collect_metrics()
            
            # 分析资源使用情况
            analysis = self.analyze_cluster_resources(metrics)
            
            # 生成报告
            report = self.generate_report(analysis)
            
            # 打印报告
            print(report)
            
            # 推送报告
            self.send_report(report)
            
        except Exception as e:
            logger.error(f"集群巡检过程中发生错误: {e}")
            error_report = f"❌ 集群巡检失败\n\n错误信息: {e}"
            self.send_report(error_report)

if __name__ == "__main__":
    inspector = ClusterInspector()
    inspector.run()